use utf8;
use strict;
use Data::Dumper;

binmode(STDOUT, ":encoding(utf-8)");

foreach my $f (@ARGV) {
  print "$f\n";
  my %p = load_pool($f, 3);
  check_pool_consistency(\%p, 3);
  print "\n";
}

die;


print STDOUT "19 by Shadow (original)\n";
my %p19_Shadow = load_pool("data/moderation_pool_19_Shadow.tsv", 3);
check_pool_consistency(\%p19_Shadow, 3);

print STDOUT "19 by GranD (cross)\n";
my %p19_GranD = load_pool("data/crossmoderation_pool_19_GranD.tsv", 3);
check_pool_consistency(\%p19_GranD, 3);

print STDOUT "19 by protoev (cross)\n";
my %p19_protoev = load_pool("data/crossmoderation_pool_19_protoev.tsv", 3);
check_pool_consistency(\%p19_protoev, 3);

print "19 Shadow vs 19 GranD:\n";
check_inter_moderator(\%p19_Shadow, \%p19_GranD);

print "19 Shadow vs 19 protoev:\n";
check_inter_moderator(\%p19_Shadow, \%p19_protoev);


print STDOUT "46 by protoev (original)\n";
my %p46_protoev = load_pool("data/moderation_pool_46_protoev.tsv", 3);
check_pool_consistency(\%p46_protoev, 3);

print STDOUT "46 by GranD (cross)\n";
my %p46_GranD = load_pool("data/crossmoderation_pool_46_GranD.tsv", 3);
check_pool_consistency(\%p46_GranD, 3);

print "46 protoev vs 46 GranD:\n";
check_inter_moderator(\%p46_protoev, \%p46_GranD);


print STDOUT "23 by GranD (original)\n";
my %p23_GranD = load_pool("data/moderation_pool_23_GranD.tsv", 3);
check_pool_consistency(\%p23_GranD, 3);

print STDOUT "23 by protoev (cross)\n";
my %p23_protoev = load_pool("data/moderation_pool_23_GranD.tsv", 3);
check_pool_consistency(\%p23_protoev, 3);

print "23 GranD vs 23 protoev:\n";
check_inter_moderator(\%p23_GranD, \%p23_protoev);

# just test
#print STDOUT "168 by protoev (original)\n";
#my %p168_protoev = load_pool("data/moderation_pool_168_protoev.tsv", 3);
#check_pool_consistency(\%p168_protoev, 3);

#print STDOUT "553 by protoev (original)\n";
#my %p553_protoev = load_pool("data/moderation_pool_553_protoev.tsv", 3);
#check_pool_consistency(\%p553_protoev, 3);

#print STDOUT "449 by protoev (original)\n";
#my %p449_protoev = load_pool("data/moderation_pool_449_protoev.tsv", 3);
#check_pool_consistency(\%p449_protoev, 3);

#print STDOUT "454 by protoev (original)\n";
#my %p454_protoev = load_pool("data/moderation_pool_454_protoev.tsv", 3);
#check_pool_consistency(\%p454_protoev, 3);

#print STDOUT "550 by protoev (original)\n";
#my %p550_protoev = load_pool("data/moderation_pool_550_protoev.tsv", 3);
#check_pool_consistency(\%p550_protoev, 3);

print STDOUT "15 by GranD (original)\n";
my %p15_GranD = load_pool("data/moderation_pool_15_GranD.tsv", 3);
check_pool_consistency(\%p15_GranD, 3);

#print STDOUT "36 by Shadow (original)\n";
#my %p36_Shadow = load_pool("data/moderation_pool_36_Shadow.tsv", 3);
#check_pool_consistency(\%p36_Shadow, 3);


sub check_inter_moderator {
  my ($rp1, $rp2) = @_;
  my $total = 0;
  my $n_decision_disagreement = 0;
  my $n_status_disagreement = 0;

  foreach my $sid (keys %{$rp1}) {
    if (! exists $rp2->{$sid}) {
      print "ERROR\t$sid\texists only in p1\n";
    }

    $total += 1;

    if ($rp1->{$sid}->{good} ne $rp2->{$sid}->{good}) {
      $n_decision_disagreement += 1;
      print "WARNING\t$sid\tdecision disagreement\t" . $rp1->{$sid}->{good} . "\t" . $rp2->{$sid}->{good} . "\n";
    } else {
      # both agree in decision
      if ($rp1->{$sid}->{status} ne $rp2->{$sid}->{status}) {
        # status difference
#        $n_status_disagreement += 1;
#        print "WARNING\t$sid\tstatus disagreement\t\"" . $rp1->{$sid}->{status} . "\"\t\"" . $rp2->{$sid}->{status} . "\"\n";
      }
    }
  }

  print "STAT\ttotal samples\t$total\n";
  print "STAT\tdecision disagreement\t$n_decision_disagreement\n";
  print "STAT\tstatus disagreement\t$n_status_disagreement\n";

  print "\n";
}

sub check_pool_consistency {
  my ($rp, $nannotators) = @_;
  my $n_samples = 0;
  my $n_full_interannotator_agreement = 0;
  my $n_full_agreement = 0;
  my $n_moderator_disagree = 0;
  my $n_moderator_agree_with_majority = 0;
  my $n_moderator_agree_with_minority = 0;

  foreach my $sid (keys %{$rp}) {
    $n_samples += 1;
  
    my %vote;
    my $total = 0;
    foreach my $a (@{$rp->{$sid}->{user}}) {
      $vote{$a} += 1;
      $total += 1;
    }
   
    # test 1: number of annotators' answers
    if ($total != $nannotators) {
      print STDOUT "ERROR\t$sid\texpected $nannotators answers but only $total available\n";
    }

    my @v = keys %vote;
    if (0 == $#v) {
      #print Dumper(\@v) . "\n";

      # 100% interannotator agreement
      $n_full_interannotator_agreement += 1;
 
      # test 2:
      if ($v[0] eq $rp->{$sid}->{good}) {
        # full agreement: all annotators + moderator
        $n_full_agreement += 1;
      } else {
        # full interannotator agreement but moderator disagree
        $n_moderator_disagree += 1;
        print STDOUT "WARNING\t$sid\tmoderator disagrees\t$v[0]\t" . $rp->{$sid}->{good} . "\n";
      }
    } elsif (1 == $#v) {
      # two variants
      my $majority_vote = undef;
      if ($vote{$v[0]} > $vote{$v[1]}) {
        $majority_vote = $v[0];
      } else {
        $majority_vote = $v[1];
      }

      # test 3:
      if ($majority_vote eq $rp->{$sid}->{good}) {
        # moderator agree with majority
        $n_moderator_agree_with_majority += 1;
      } else {
        $n_moderator_agree_with_minority += 1;
      }
    }
  }

  print STDOUT "STAT\ttotal samples\t$n_samples\n";
  print STDOUT "STAT\tfull interannotator agreement\t$n_full_interannotator_agreement\n";
  print STDOUT "STAT\tfull agreement\t$n_full_agreement\n";
  print STDOUT "STAT\tmoderator disagrees\t$n_moderator_disagree\n";
  print STDOUT "STAT\tmoderator agree with majority\t$n_moderator_agree_with_majority\n";
  print STDOUT "STAT\tmoderator agree with minority\t$n_moderator_agree_with_minority\n";
  print STDOUT "\n";
}

sub load_pool {
  my ($fn, $nannotators) = @_;
  my %h;

  open(F, "< $fn") || die "Can't open \"$fn\"";
  binmode(F, ":encoding(utf-8)");
  while (<F>) {
    chomp $_;
    $_ =~ s/\t /\t/g;
    my ($sample_id, @answer) = split(/\t/, $_);
    for (my $i = 0; $i < $nannotators; $i++) {
      $h{$sample_id}->{user}->[$i] = $answer[$i];
    }
    $h{$sample_id}->{good} = $answer[$nannotators];
    $h{$sample_id}->{status} = $answer[$nannotators+1];
  }
  close(F);
  return %h;
}
